{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# GPU实践 + 在CNN图像分类模型上实践"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0.0\n",
      "sys.version_info(major=3, minor=6, micro=10, releaselevel='final', serial=0)\n",
      "matplotlib 3.1.2\n",
      "numpy 1.18.1\n",
      "pandas 0.25.3\n",
      "sklearn 0.22.1\n",
      "tensorflow 2.0.0\n",
      "tensorflow_core.keras 2.2.4-tf\n"
     ]
    }
   ],
   "source": [
    "# 导入\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "\n",
    "print(tf.__version__)\n",
    "print(sys.version_info)\n",
    "for module in mpl,np,pd,sklearn,tf,keras:\n",
    "    print(module.__name__,module.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 设置GPU"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GPU实践1：内存增长和可见"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "# 打印日志\n",
    "tf.debugging.set_log_device_placement(True)\n",
    "\n",
    "# 获取所有物理GPU\n",
    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "# 设置某个GPU可见\n",
    "tf.config.experimental.set_visible_devices(gpus[0], 'GPU')\n",
    "# 设置内存增长\n",
    "for gpu in gpus:\n",
    "    tf.config.experimental.set_memory_growth(gpu, True)\n",
    "print(len(gpus))\n",
    "\n",
    "# 获取所有逻辑GPU\n",
    "logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n",
    "print(len(logical_gpus))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GPU实践2：为GPU做逻辑切分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "2\n"
     ]
    }
   ],
   "source": [
    "tf.debugging.set_log_device_placement(True)\n",
    "\n",
    "# 获取所有物理GPU\n",
    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "# 设置某个GPU可见\n",
    "tf.config.experimental.set_visible_devices(gpus[0], 'GPU')\n",
    "# 逻辑切分\n",
    "tf.config.experimental.set_virtual_device_configuration(\n",
    "    gpus[0],\n",
    "    [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=2048),\n",
    "     tf.config.experimental.VirtualDeviceConfiguration(memory_limit=2048)]\n",
    ")\n",
    "print(len(gpus))\n",
    "\n",
    "# 获取所有逻辑GPU\n",
    "logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n",
    "print(len(logical_gpus))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GPU实践3：手工指定的方式使用多GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "2\n"
     ]
    }
   ],
   "source": [
    "tf.debugging.set_log_device_placement(True)\n",
    "tf.config.set_soft_device_placement(True)\n",
    "\n",
    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "tf.config.experimental.set_virtual_device_configuration(\n",
    "    gpus[0],\n",
    "    [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=2048),\n",
    "     tf.config.experimental.VirtualDeviceConfiguration(memory_limit=2048)]\n",
    ")\n",
    "print(len(gpus))\n",
    "\n",
    "logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n",
    "print(len(logical_gpus))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op MatMul in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "/job:localhost/replica:0/task:0/device:GPU:1\n",
      "Executing op MatMul in device /job:localhost/replica:0/task:0/device:GPU:1\n",
      "Executing op AddN in device /job:localhost/replica:0/task:0/device:CPU:0\n",
      "tf.Tensor(\n",
      "[[ 44.  56.]\n",
      " [ 98. 128.]], shape=(2, 2), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# 手工指定，进行矩阵计算\n",
    "c = []\n",
    "for gpu in logical_gpus:\n",
    "    print(gpu.name)\n",
    "    with tf.device(gpu.name):\n",
    "        a = tf.constant([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])\n",
    "        b = tf.constant([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])\n",
    "        c.append(tf.matmul(a, b))\n",
    "\n",
    "with tf.device('/CPU:0'):\n",
    "    matmul_num = tf.add_n(c)\n",
    "\n",
    "print(matmul_num)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据读取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5000, 28, 28) (5000,)\n",
      "(55000, 28, 28) (55000,)\n",
      "(10000, 28, 28) (10000,)\n"
     ]
    }
   ],
   "source": [
    "# # 导入数据集fashion mnist\n",
    "fashion_mnist = keras.datasets.fashion_mnist\n",
    "(x_train_all,y_train_all),(x_test,y_test) = fashion_mnist.load_data()\n",
    "x_valid,x_train = x_train_all[:5000],x_train_all[5000:]\n",
    "y_valid,y_train = y_train_all[:5000],y_train_all[5000:]\n",
    "\n",
    "print(x_valid.shape,y_valid.shape)\n",
    "print(x_train.shape,y_train.shape)\n",
    "print(x_test.shape,y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据归一化\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "scaler = StandardScaler()\n",
    "x_train_scaled = scaler.fit_transform(\n",
    "    x_train.astype(np.float32).reshape(-1,1)).reshape(-1,28,28,1)\n",
    "x_valid_scaled = scaler.transform(\n",
    "    x_valid.astype(np.float32).reshape(-1,1)).reshape(-1,28,28,1)\n",
    "x_test_scaled = scaler.transform(\n",
    "    x_test.astype(np.float32).reshape(-1,1)).reshape(-1,28,28,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Executing op TensorSliceDataset in device /job:localhost/replica:0/task:0/device:CPU:0\n",
      "Executing op AnonymousRandomSeedGenerator in device /job:localhost/replica:0/task:0/device:CPU:0\n",
      "Executing op ShuffleDatasetV2 in device /job:localhost/replica:0/task:0/device:CPU:0\n",
      "Executing op RepeatDataset in device /job:localhost/replica:0/task:0/device:CPU:0\n",
      "Executing op BatchDatasetV2 in device /job:localhost/replica:0/task:0/device:CPU:0\n",
      "Executing op PrefetchDataset in device /job:localhost/replica:0/task:0/device:CPU:0\n"
     ]
    }
   ],
   "source": [
    "# 生成dataset\n",
    "def make_dataset(images, labels, epochs, batch_size, shuffle=True):\n",
    "    dataset = tf.data.Dataset.from_tensor_slices((images, labels))\n",
    "    if shuffle:\n",
    "        dataset = dataset.shuffle(10000)\n",
    "    dataset = dataset.repeat(epochs).batch(batch_size).prefetch(50)\n",
    "    return dataset\n",
    "\n",
    "batch_size = 128\n",
    "epochs = 100\n",
    "train_dataset = make_dataset(x_train_scaled, y_train, epochs, batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNN模型搭建\n",
    "指定在某个GPU上\n",
    "\n",
    "好处：当模型很大时，model需要放在不同GPU上，但仍然是串行而非并行"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Executing op RandomUniform in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op Sub in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op Mul in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op Add in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarIsInitializedOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op LogicalNot in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op Assert in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op AssignVariableOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op Fill in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op RandomUniform in device /job:localhost/replica:0/task:0/device:GPU:1\n",
      "Executing op Sub in device /job:localhost/replica:0/task:0/device:GPU:1\n",
      "Executing op Mul in device /job:localhost/replica:0/task:0/device:GPU:1\n",
      "Executing op Add in device /job:localhost/replica:0/task:0/device:GPU:1\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:1\n",
      "Executing op VarIsInitializedOp in device /job:localhost/replica:0/task:0/device:GPU:1\n",
      "Executing op LogicalNot in device /job:localhost/replica:0/task:0/device:GPU:1\n",
      "Executing op Assert in device /job:localhost/replica:0/task:0/device:GPU:1\n",
      "Executing op AssignVariableOp in device /job:localhost/replica:0/task:0/device:GPU:1\n",
      "Executing op Fill in device /job:localhost/replica:0/task:0/device:GPU:1\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:1\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:1\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:1\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op LogicalNot in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op Assert in device /job:localhost/replica:0/task:0/device:GPU:0\n"
     ]
    }
   ],
   "source": [
    "model = keras.models.Sequential()\n",
    "# 构建模型\n",
    "with tf.device(logical_gpus[0].name):\n",
    "    model.add(keras.layers.Conv2D(filters=32, kernel_size=3, padding='same',\n",
    "                                  activation='relu', input_shape=(28,28,1)))\n",
    "    model.add(keras.layers.Conv2D(filters=32, kernel_size=3, padding='same', activation='relu'))\n",
    "    model.add(keras.layers.MaxPool2D(pool_size=2))\n",
    "\n",
    "    model.add(keras.layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\n",
    "    model.add(keras.layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu'))\n",
    "    model.add(keras.layers.MaxPool2D(pool_size=2))\n",
    "\n",
    "    model.add(keras.layers.Conv2D(filters=128, kernel_size=3, padding='same', activation='relu'))\n",
    "    model.add(keras.layers.Conv2D(filters=128, kernel_size=3, padding='same', activation='relu'))\n",
    "    model.add(keras.layers.MaxPool2D(pool_size=2))\n",
    "\n",
    "with tf.device(logical_gpus[1].name):\n",
    "    model.add(keras.layers.Flatten())\n",
    "    model.add(keras.layers.Dense(128, activation='relu'))\n",
    "    model.add(keras.layers.Dense(10, activation='softmax'))\n",
    "\n",
    "# 模型编译，固化模型\n",
    "model.compile(loss='sparse_categorical_crossentropy',\n",
    "              optimizer='adam',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d (Conv2D)              (None, 28, 28, 32)        320       \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 28, 28, 32)        9248      \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 14, 14, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 14, 14, 64)        18496     \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 14, 14, 64)        36928     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 7, 7, 64)          0         \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 7, 7, 128)         73856     \n",
      "_________________________________________________________________\n",
      "conv2d_5 (Conv2D)            (None, 7, 7, 128)         147584    \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 3, 3, 128)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 1152)              0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 128)               147584    \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 435,306\n",
      "Trainable params: 435,306\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Executing op DatasetCardinality in device /job:localhost/replica:0/task:0/device:CPU:0\n",
      "Train for 429 steps\n",
      "Epoch 1/10\n",
      "Executing op OptimizeDataset in device /job:localhost/replica:0/task:0/device:CPU:0\n",
      "Executing op ModelDataset in device /job:localhost/replica:0/task:0/device:CPU:0\n",
      "Executing op AnonymousIteratorV2 in device /job:localhost/replica:0/task:0/device:CPU:0\n",
      "Executing op MakeIterator in device /job:localhost/replica:0/task:0/device:CPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op AssignVariableOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op Fill in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:1\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:1\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:1\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:1\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:1\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:1\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:1\n",
      "Executing op VarHandleOp in device /job:localhost/replica:0/task:0/device:GPU:1\n",
      "Executing op __inference_initialize_variables_1012 in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "Executing op __inference_distributed_function_1345 in device /job:localhost/replica:0/task:0/device:GPU:0\n",
      "429/429 [==============================] - 29s 67ms/step - loss: 0.4959 - accuracy: 0.8197\n",
      "Epoch 2/10\n",
      "429/429 [==============================] - 23s 53ms/step - loss: 0.2776 - accuracy: 0.8986\n",
      "Epoch 3/10\n",
      "429/429 [==============================] - 23s 53ms/step - loss: 0.2293 - accuracy: 0.9160\n",
      "Epoch 4/10\n",
      "429/429 [==============================] - 23s 53ms/step - loss: 0.1967 - accuracy: 0.9282\n",
      "Epoch 5/10\n",
      "429/429 [==============================] - 23s 53ms/step - loss: 0.1673 - accuracy: 0.9386\n",
      "Epoch 6/10\n",
      "429/429 [==============================] - 23s 53ms/step - loss: 0.1480 - accuracy: 0.9453\n",
      "Epoch 7/10\n",
      "429/429 [==============================] - 23s 53ms/step - loss: 0.1289 - accuracy: 0.9528\n",
      "Epoch 8/10\n",
      "429/429 [==============================] - 23s 53ms/step - loss: 0.1111 - accuracy: 0.9592\n",
      "Epoch 9/10\n",
      "429/429 [==============================] - 23s 53ms/step - loss: 0.0956 - accuracy: 0.9646\n",
      "Epoch 10/10\n",
      "429/429 [==============================] - 23s 53ms/step - loss: 0.0829 - accuracy: 0.9693\n",
      "Executing op DeleteIterator in device /job:localhost/replica:0/task:0/device:CPU:0\n"
     ]
    }
   ],
   "source": [
    "# 训练\n",
    "history = model.fit(\n",
    "    train_dataset, steps_per_epoch = x_train_scaled.shape[0] // batch_size, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeYAAAEzCAYAAADkYKBTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3de3zU1Z3/8dfJzGQmySQBcuOuQJCbggh4QUtDbRGrXS/VVdtVpPXWarvbbtfWrt3+um1/264/3W27tpSi1V5sl3rZui1qS2uk1BsgoCKC4SbhlhDIZXK/nN8f30kyCSGZQJLvdybv5+ORR2a+czLzOUF4e873fM/XWGsRERERb0hxuwARERHppGAWERHxEAWziIiIhyiYRUREPETBLCIi4iEKZhEREQ/pM5iNMY8aY8qMMW+f5HVjjPm+MabEGPOmMea8gS9TRERkeIhnxPwYsLSX1y8Hpka/7gB+dPpliYiIDE99BrO1dh1wrJcmVwE/s45XgRHGmDEDVaCIiMhwMhDnmMcB+2Oel0aPiYiISD/5B+A9TA/Hetzn0xhzB850N6FQaN7EiRMH4OPd1dbWRkpKYq+hS4Y+gPrhJcnQB0iOfiRDHyB5+rFz586j1tq83toMRDCXAhNino8HDvbU0Fq7ElgJMG3aNLtjx44B+Hh3FRcXU1RU5HYZpyUZ+gDqh5ckQx8gOfqRDH2A5OmHMWZfX20G4n8/ngVuia7OvhCostYeGoD3FRERGXb6HDEbY34FFAG5xphS4OtAAMBauwJYA3wUKAHqgOWDVayIiEiy6zOYrbU39fG6Be4esIpERESGscQ/ky4iIpJEFMwiIiIeMhCrskVERIaHtlZoaYTWJufrhMfN0Bo91tIUfdzc2S4OCmYREfGe1hZoaYgGWiOh+sNQvqNb4PX0+GSBGftavD/TQ8jatkHvuoJZREQ6tbV2BmJLo/O4tSl6LPq9Nea1jmMD3KZbAF4I8Nop9inFD75U58sf7HzsSwV/Kviix0JZ0ceBru380WO+YNfHHe1O9riHn/nG2D7LVTCLiHiNtdHAqofmhpjv0a/m+pN8bzjhZ2YefB8OrewWjL2Erm09/fpNCvhD0VALOeHnD3UGlz/khGBfbdqP+VLZXrKHGbNm9xCswZ5D1h/TJsV3+n0aQgpmEZG+tLX2Eoan+v3EEO0SwD3vbNw344NAmhNogTTCTa2QMsoJMH8IUsOQntsl9DoCsb1N97DseN7TsfbnsT8/8NFyJFLMjHOKBvx9vUjBLCKJrT00m+ucr6a6rs87jrV/1UNTbQ8/U3fCsUsaamBdC7Q1n3p97YHWHpb+EARC4E+DUDaER3c+D3RreyrffYEuH/96kmxlOZwomEVk8FjrTJX2Gpj10Fzbr8Ds8jMtDf2vqz3EAhnO99R0CKQ706uZo53HgTQOHznG+EmFMaHZ1/fuIRmEJLjxggwtBbPIcNTaHA23+uj0aX1nyDXXOdOpvb5W1zkt2/21lnouqq2Cl1ucY/09Z2lSOgMzkAap7Y/TndFle4hGw7PL64H0mNd7CN7243GecywpLma8RpsyxBTMIl7R1gpNEVIbK+DY7n6EY2cgOu16Cs5ur53qAh9f0BkZBtJjRp1pzogxPbdj9FhxtJKxZxR2vt4RkCcL0ZiQ9aWC6elusiLDg4JZ5HRZ64ReYw00VjtfDdWdz2Mf9/ZaUwSAhQCv9OPz26dRO8IyvXNaNVxw8iANxE7DpnV77STvF+dIc2dxMWM10hQ5JQpmGd5aW6KB2c8Q7f68raXvzwpkQDDTOY8ZzHIeZ411vgezneOpYXbsLWXarHO7hWJscIa6PtboUiSpKJglcbW2QEMVNFRCfSUjj22GbZWdQdtQ3XfANtf1/Tkp/s4gbQ/VrPGQHz3W5bXsmMfR10JZkJoZ9yUkh5qKmTan6PR+NyKSsBTM4q62Vidc6493BGzH956ONVRCfbR9U02Xt5oD8Ga3908Ndw3IUDZkT4iGZ3a3UI1pF4wZ1QbSNCoVkSGjYJbT19YGjVV9hOnxngO2sar39/YFIW0EpI2E0AjIGgf5s5xjoRFdXtu8fRdzL/xg16niBNvxR0REwSyOtrboNG9fYdpD+DZU0+suRb7UriGaOQbyZ3Qe6xawXY4F0uLuQtWhYhh99mn/KkRE3KRgTmbWOiEaOeJ81RzpfBw5AjWHIVLGxZUH4aXa3u+akhLoGpjhAsid1vXYyQJWU8EiInFTMCeilkaIlHUL2e6hG/3e01aC/jTILHDCNW8aZamTGVd4du8BG0hXuIqIDAEFs1dY60wLdw/YnkK3/njP75Ge62wnGM6H3LOc4A0XdIZwOPpaMLNLyL5XXMw4XXMqIuIJCubB1tIEtWXdArYMIs40cvt0MpEjzr1Ju2vfJCJcALlT4cxLOgM2NnQz8k7YvF5ERBKPgvl0VB9ixPE34a2jXc7Zdgnd+mM9/2x6Tmfg5hTGjGoLuo50g1maQhYRGUYUzP1lLez9C7y6Anas4VwsbI2+5gt2BuyoyTDxom5Tye1f+RrdiohIjxTM8WquhzdXw2s/hrJtzoj3A//Ilupszr3kMidwQ9ka3YqIyGlRMPelqhQ2rIJNjzmLrgrOgasehrOvg0CIyuJiyJvmdpUiIpIkFMw9sRb2vwavrYB3ngUsTL8CLvgMnLFQo2IRERk0CuZYLY3w9tNOIB/a4kxNX3Q3LLgNRp7hdnUiIjIMKJjBuYxp46POV20Z5E2HK/8DZt/g3LhdRERkiAzvYD7whjM6fvtp5366Z10GF9wJkxdrulpERFwx/IK5tRm2P+tc7lT6unOf3AWfhvPvgJwpblcnIiLD3PAJ5toK2PRT2PAI1Bx0rjNe+l049xPObQJFREQ8IPmD+fDb8NqP4M3fOFteTl4MH/tPKPwIpKS4XZ2IiEgXyRnMba2wY42zGcjevzh3Rpr7STj/Tsif7nZ1IiIiJ5VcwVx/HDb/Al5fCZXvQ/YE+Mi/wnm3OLcwFBER8bjkCObync7q6q2/guY6OOMSWPJtmPZR8CVHF0VEZHhI3NRqa4OStc75411/dm4gcc71zuVOY2a7XZ2IiMgpSbxgbqyBLU8454+P7YLMMfCh+2HecsjIdbs6ERGR05I4wXxsN7y20jmH3FQD4xfA4q/CzKt0C0UREUka3g5ma2F3sTM63vk8pPhh1jVwwV0wfp7b1YmIiAw4bwZzUx28+d9OIJdvh/RcWPRPzg5dmaPdrk5ERGTQeCuYK9+P3vv4cWiohNGz4eofwaxrIRByuzoREZFB534wWwvvvwKv/gje/R1gYMaVzr2PJ16om0mIiMiw4lowGyxs/qVz/fHhNyE0AhZ+3rn38YgJbpUlIiLiKteCOSOyF377WcibAR/7Hpzzt5Ca7lY5IiIinuBaMLf6QnDLb2HSBzVdLSIiEuVaMNenjYHJRW59vIiIiCfFdd9DY8xSY8wOY0yJMeYrPbyebYz5X2PMVmPMNmPM8oEvVUREJPn1GczGGB/wMHA5MBO4yRgzs1uzu4F3rLVzgCLgQWNM6gDXKiIikvTiGTGfD5RYa3dba5uAXwNXdWtjgUxjjAHCwDGgZUArFRERGQaMtbb3BsZcByy11t4WfX4zcIG19p6YNpnAs8B0IBO4wVr7+x7e6w7gDoC8vLx5q1evHqh+uCYSiRAOh90u47QkQx9A/fCSZOgDJEc/kqEPkDz9WLx48SZr7fze2sSz+KunJdPd0/wyYAvwIWAK8EdjzF+stdVdfsjalcBKgGnTptmioqI4Pt7biouLSfR+JEMfQP3wkmToAyRHP5KhD5A8/YhHPFPZpUDsjh/jgYPd2iwHnraOEmAPzuhZRERE+iGeYN4ATDXGTIou6LoRZ9o61vvApQDGmAJgGrB7IAsVEREZDvqcyrbWthhj7gFeAHzAo9babcaYu6KvrwC+CTxmjHkLZ+r7y9bao4NYt4iISFKKa4MRa+0aYE23YytiHh8ElgxsaSIiIsNPXBuMiIiIyNBQMIuIiHiIgllERMRDFMwiIiIeomAWERHxEAWziIiIhyiYRUREPETBLCIi4iEKZhEREQ9RMIuIiHiIgllERMRDFMwiIiIeomAWERHxEAWziIiIhyiYRUREPETBLCIi4iEKZhEREQ9RMIuIiHiIgllERMRDFMwiIiIeomAWERHxEAWziIiIhyiYRUREPETBLCIi4iEKZhEREQ9RMIuIiHiIgllERMRDFMwiIiIeomAWERHxEAWziIiIhyiYRUREPETBLCIi4iEKZhEREQ9RMIuIiHiIgllERMRDFMwiIiIeomAWERHxEAWziIiIhyiYRUREPETBLCIi4iEKZhEREQ9RMIuIiHiIgllERMRDFMwiIiIeomAWERHxkLiC2Riz1BizwxhTYoz5yknaFBljthhjthljXhrYMkVERIYHf18NjDE+4GHgI0ApsMEY86y19p2YNiOAHwJLrbXvG2PyB6tgERGRZBbPiPl8oMRau9ta2wT8GriqW5tPAE9ba98HsNaWDWyZIiIiw0M8wTwO2B/zvDR6LNZZwEhjTLExZpMx5paBKlBERGQ4Mdba3hsYcz1wmbX2tujzm4HzrbWfi2nzX8B84FIgDXgFuMJau7Pbe90B3AGQl5c3b/Xq1QPYFXdEIhHC4bDbZZyWZOgDqB9ekgx9gOToRzL0AZKnH4sXL95krZ3fW5s+zzHjjJAnxDwfDxzsoc1Ra20tUGuMWQfMAboEs7V2JbASYNq0abaoqCiOj/e24uJiEr0fydAHUD+8JBn6AMnRj2ToAyRPP+IRz1T2BmCqMWaSMSYVuBF4tlub3wIfMMb4jTHpwAXA9oEtVUREJPn1OWK21rYYY+4BXgB8wKPW2m3GmLuir6+w1m43xjwPvAm0AaustW8PZuEiIiLJKJ6pbKy1a4A13Y6t6Pb8AeCBgStNRERk+NHOXyIiIh6iYBYREfEQBbOIiIiHKJhFREQ8RMEsIiLiIQpmERERD1Ewi4iIeIiCWURExEMUzCIiIh6iYBYREfEQBbOIiIiHKJhFREQ8RMEsIiLiIQpmERERD1Ewi4iIeIiCWURExEMUzCIiIh6iYBYREfEQBbOIiIiHKJhFREQ8RMEsIiLiIQpmERERD3EtmI/UWVpa29z6eBEREU9yLZjrWyzf+v12tz5eRETEk1wL5qxUw2Mv7+Xnr+5zqwQRERHPcS2YR4UMH5qez/95dhvr3zvqVhkiIiKe4urir+/deC6FeWE++8tN7CqPuFmKiIiIJ7gazJmhAKuWzSfgS+G2xzdSWdfkZjkiIiKuc/1yqQmj0llx8zwOHK/ns798g2at1BYRkWHM9WAGWHDmKP7vtefw8q4Kvv7sNqy1bpckIiLiCr/bBbS7bt54SsoirHhpF2flh7n14klulyQiIjLkPBPMAPdeNo1d5RH+9XfvcGZuBkXT8t0uSUREZEh5Yiq7XUqK4T9vOJdpo7P43BObKSmrcbskERGRIeWpYAbICPpZtWw+wYCPTz22keO1WqktIiLDh+eCGWDciDRW3jKPw9UN3PWLTTS1aKW2iIgMD54MZoDzJo7kgetm89qeY3ztf97WSm0RERkWPLX4q7urzh1HSVmEH/y5hKkFYW77wGS3SxIRERlUng5mgC98+CxKyiJ8e812JuVmcOmMArdLEhERGTSencpul5JiePBv5zBrbBaf/9VmdhzWSm0REUleng9mgPRUPz+5ZT4ZQT+ffnwDRyONbpckIiIyKBIimAHGZKfxk1vmU17TyF0/30RjS6vbJYmIiAy4hAlmgDkTRvDg385h477j3Pf0W1qpLSIiScfzi7+6u3L2WHaV1fIfa3dyVkEmd31witsliYiIDJiEC2aAz19aSEl5hO8+/y6TczNYMmu02yWJiIgMiISaym5njOGB62Yze1w2//DfW3jnYLXbJYmIiAyIuILZGLPUGLPDGFNijPlKL+0WGGNajTHXDVyJPQsFfPzklvlkhQLc9vgGymoaBvsjRUREBl2fwWyM8QEPA5cDM4GbjDEzT9Luu8ALA13kyeRnhVi1bD7H65q58+ebaGjWSm0REUls8YyYzwdKrLW7rbVNwK+Bq3po9zngKaBsAOvr09njsvmPG+aw+f1KvvzUm1qpLSIiCS2eYB4H7I95Xho91sEYMw64BlgxcKXFb+nZY/iny6bx2y0HefjFEjdKEBERGRCmrxGmMeZ64DJr7W3R5zcD51trPxfT5jfAg9baV40xjwG/s9Y+2cN73QHcAZCXlzdv9erVA9YRay0r32zklUOt3H1ukAWjh2bBeSQSIRwOD8lnDZZk6AOoH16SDH2A5OhHMvQBkqcfixcv3mStnd9rI2ttr1/ARcALMc/vA+7r1mYPsDf6FcGZzr66t/c966yz7ECrb2qxVz+83k67f419q7RywN+/Jy+++OKQfM5gSoY+WKt+eEky9MHa5OhHMvTB2uTpB7DR9pG78UxlbwCmGmMmGWNSgRuBZ7uF+yRr7ZnW2jOBJ4HPWmv/J77/fxg4oYCPlTfPJycjyG2Pb+RItVZqi4hIYukzmK21LcA9OKuttwOrrbXbjDF3GWPuGuwC+ysvM8iqZfOpbmjm9p9tpL5JK7VFRCRxxHUds7V2jbX2LGvtFGvtt6PHVlhrT1jsZa291fZwfnkozRiTxfdunMtbB6r40pNbtVJbREQSRkLu/BWPj8ws4CtLp/P7Nw/xvT+953Y5IiIicUnIvbLjdceiybxXFuE/177HlLwwH5sz1u2SREREepW0I2Zw9tT+9jVns+DMkXzpN1vZur/S7ZJERER6ldTBDBD0+1jxd/PIywxy+882cqiq3u2SRERETirpgxkgJxzkkWULqGtq5bbHN1LX1OJ2SSIiIj0aFsEMMG10Jj+4aS7bD1Xzxf/eSlubVmqLiIj3DJtgBlg8PZ+vfnQGz287zEN/3Ol2OSIiIidI6lXZPfn0JZPYVR7hv14soTA/zNVzx/X9QyIiIkNkWI2YwVmp/Y2/OZsLJ4/i3qfeZNO+426XJCIi0mHYBTNAqj+FH31yHmOyQ9z5842UHq9zuyQRERFgmAYzwMiMVB5ZtoDGljZue3wjtY1aqS0iIu4btsEMUJgf5uFPnMd7ZRH+/tdbtFJbRERcN6yDGWDRWXn8y5UzWbv9CP/+wg63yxERkWFu2K3K7sktF53Be2U1rHhpF4X5Ya6bN97tkkREZJga9iNmcFZqf/1js7i4MIf7nn6TDXuPuV2SiIgMUwrmqIAvhR9+Yh4TRqZz5883sf+YVmqLiMjQUzDHyE4PsGrZfFrbLJ9+fAM1Dc1ulyQiIsOMgrmbyXlhfvTJ89hdXsvnf7WZVq3UFhGRIaRg7sHCwly+cdUsXtxRzr+t2e52OSIiMoxoVfZJfPKCM3jvSIRV6/dQmB/mxvMnul2SiIgMAxox9+L+K2aw6Kw87v+ft3llV4Xb5YiIyDCgYO6F35fCf31iLmfmZvCZX25iX0Wt2yWJiEiSUzD3ISsU4JFl8wH41GMbqNZKbRERGUQK5jickZPBir+bx76KOu55YjMtrW1ulyQiIklKwRynCyfn8O1rzmbdznK+9Xut1BYRkcGhVdn9cMOCiV1Wav/dhWe4XZKIiCQZBXM/3ffRGew+WsvXn93GmTkZbpcjIiJJRlPZ/eRLMXzvxnMpzAvz2V9u4s3yFhqaW90uS0REkoSC+RRkhpw9tYMBHw9tamTON/7AzY+8xoqXdvH2gSratI2niIicIk1ln6IJo9Ip/lIRP/ltMdVpY/lryVG+89y7AIxMD7BwSi4LC3O4pDCXiaPSMca4XLGIiCQCBfNpyAj6OTffT1HRTADKahp4uaSCv5YcZX3JUX7/1iEAxo9M45LCXC4uzGXhlBxywkE3yxYREQ9TMA+g/MwQV88dx9Vzx2GtZc/R2i4h/esN+wGYMSaLSwpzuLgwl/MnjSI9VX8MIiLiUCIMEmMMk/PCTM4Lc/NFZ9LaZnnrQBV/LTnKX0uO8vjL+/jJX/YQ8BnOmzjSGVFPzWX2uGz8Pp36FxEZrhTMQ8SXYjh3wgjOnTCCuxcXUt/UysZ9x1gfDeqH1u7kwT/uJDPo54LJOVxSmMMlU3OZkhfW+WkRkWFEweyStFQfH5iaxwem5gFwrLaJV3ZV8NddTlCv3X4EgIKsIBcX5nacoy7ICrlZtoiIDDIFs0eMykjlitljuGL2GAD2H6vrOD9dvKOcp984AEBhfrgjpC+YPIqsUMDNskVEZIApmD1qwqh0bjx/IjeeP5G2Nsv2w9W8XFLB+pKj/PeG/Tz28l58KYY547O5pDCXhYW5zJ04gqDf53bpIiJyGhTMCSAlxTBrbDazxmZz+6LJNLa0svn9yo4R9X+9WML3/1xCWsDH+ZNGdYyop4/OJCVF56dFRBKJgjkBBf0+Lpycw4WTc/jHJdOobmjm1V0VvLzLGVF/e41z96ucjFQWFuZySWEOC6fkMmFUusuVi4hIXxTMSSArFGDJrNEsmTUagMNVDR2XZa0vOcr/bj0IwBk56R0LyS6anMPIjFQ3yxYRkR4omJPQ6OwQH583no/PG4+1ll3lEda/d5T1JRU8u+UgT7z2PsbA2WOzWViYQzjSwtmRRnK1I5mIiOsUzEnOGENhfiaF+ZncevEkWlrb2Fpa1TGafnT9HppbLQ9uWktuOMiMMZlMK8hk+pgspo/OpDA/TCigBWUiIkNFwTzM+H0pzDtjJPPOGMnnL51KXVMLj/3vSwQLJvPuoWrePVzDz1/dR2NLG+BsjDIpN4PpozOZMSYrGtqZjBuRpo1PREQGgYJ5mEtP9TMzx0fRJZM6jrW2WfZW1PLuoRrePeyE9dbSSn735qGONplBP9NGOyE9fXQWM8ZkclZBJpm6rlpE5LQomOUEvhTDlLwwU/LCHRueAEQaW9hxOBrWh2rYcbiG3245yC8a3u9oM35kGtNHO9Pg7aF9Zk669v8WEYmTglniFg76O6bB21lrOVjVwI7D1Ww/VMO7h2vYcbiaF3eU0dpmAUj1p3BWQbgzsEdnMX1MphabiYj0IK5gNsYsBb4H+IBV1trvdHv9k8CXo08jwGestVsHslDxJmMM40akMW5EGh+aXtBxvLGllZKyiDOyPlLD9kPVvLSznCc3lXa0yQ0Ho0GtxWYiIu36DGZjjA94GPgIUApsMMY8a619J6bZHuCD1trjxpjLgZXABYNRsCSGoN/XsVtZrIpIIzsO17A9OrLubbFZ7Ohai81EZLiIZ8R8PlBird0NYIz5NXAV0BHM1tqXY9q/CowfyCIleeSEgywsDLKwMLfjWPtisx2Ha3j3UDXbD9fwZmnVSRebTRudxYzRmUwbrcVmIpJ8jLW29wbGXAcstdbeFn1+M3CBtfaek7T/EjC9vX231+4A7gDIy8ubt3r16tMs332RSIRwOOx2GafFq32ob7EcqGmjNNLG/hrnq7SmjbqWzjY5IcOEzBQmZKaQE2hick4aozNSSPUl7ujaq38e/ZEMfYDk6Ecy9AGSpx+LFy/eZK2d31ubeEbMPf0L12OaG2MWA58GLunpdWvtSpxpbqZNm2aLiori+HhvKy4uJtH7kUh9sNZyqKqBd6OLzdpXia/ZW0trmwEaMAYmjkqnMC9MYX6YKflhpuY7jxNhhJ1Ifx4nkwx9gOToRzL0AZKnH/GIJ5hLgQkxz8cDB7s3MsbMBlYBl1trKwamPJGujDGMHZHG2B4Wm61+7iVGnjGdkrII75VF2FUW4S/vHaWpta2jXUFWkKn5mScEdk5Gqs5hi4gnxBPMG4CpxphJwAHgRuATsQ2MMROBp4GbrbU7B7xKkT4E/T4mZKZQNHtsl+MtrW3sP15PSVkkGtg17CqL8JuN+6ltau1oNyI9QGFemKkFzvXbhdHAHpudpltnisiQ6jOYrbUtxph7gBdwLpd61Fq7zRhzV/T1FcC/ADnAD6Ojjpa+5tBFhoLfl8Kk3Awm5WbwkZmdI2xrLYerG3jviBPYJeURSo5EeGHbEY7V7u9ol57q6xLU7V9njNKmKSIyOOK6jtlauwZY0+3YipjHtwEnLPYS8SpjDGOy0xiTncais/K6vFYRaewM6+hI+9XdFTyz+UBHm4DPuayrMD/snMsuyKQwL8zkvAxdhy0ip0U7f4l0kxMOkhMOcsHknC7HI40t7Iqev24P7HcOVvP824eJbnKW8AvPRMR9CmaROIWDfuZMGMGcCSO6HG9obmVvRe0J0+JaeCYip0LBLHKaQgFfdB/wrC7H2xeevXekpmNaPJ6FZ8cPt5D1/nFGZ4XIywwS0LlskWFFwSwySGIXni2JOd5+LXZJzLT4rrIIz799mON1zQA8vMXZTM8YZ0/xgqwgo7NCFGSFnO/ZzvfR2c6xrJBfo26RJKFgFhlisddid194dry2iWf/9BcmnHU2h6saOVzdwJGqBg5XN1B6vJ6N+45TGQ3vWGkBXzSkg12DO+axRt8iiUHBLOIhIzNSOSPLR1HM5indNTS3UlbthHZscLc/3rjvOGXVjV3Ob0Pn6Ltj5J0d+7gzxDODGn2LuEnBLJJgQgEfE3PSmZiTftI21lqO1TY5YV3d0MPou46N+471OPpOT/V1CWxn+jxIQczoOz8zqOu4RQaJglkkCRljOi776n7rzVh9jb437D12SqPvA5E2qhuaNfoWOQUKZpFhbDBH3/+8/g9dRt8FWUGd+xaJg4JZRHp1KqPvP7/6BjnjJvfr3LdWnos4FMwiMiBiR991+/wULZp8QhtrLcfrmjlcFR19Vzd0eVx6vJ5N+453XDYWKy3gc0bdsYvVYs+DZ4fICwdJ9Wv0LYnNU8Hc3NxMaWkpDQ0NbpcSt+zsbLZv3+52GQCEQiHGjx9PIKCtH8WbjDGMykhlVEYqM8dmnbRdQ3Mr5TWNXYM7OnV+pLqBN94/zpHqRppaThx952Sknjjqjj5uH5VnpwU0+hbP8lQwl5aWkpmZyZlnnpkwf2lqamrIzMx0uwystVRUVFBaWsqkSZPcLkfktIQCPiaMSmfCqN7PfVfWNZ+wcK09xA9WNbB5fyXHapt6eP+U6HnvrtPlsdPpBVmhweyiyEl5KqbhQdwAAA+XSURBVJgbGhoSKpS9xBhDTk4O5eXlbpciMiSMMYzMSGVkRiozxpx89N3Y4pz7PnHqvJEjVQ1s2V/J4W0NJ4y+ATIDMG7LOvIyg+RlBsnPdC4Vy8/q+jg91VP/lEqC89x/TQrlU6ffnciJgv74Rt9V9c3dps4b2bJjN/7MdMpqGtlVFqE80khzqz3h58NBP/nt4R29zjs/NsyznOeaQpd4eC6Y3RYOh4lEIm6XISJDyBjDiPRURqSndrkZSbH/AEVF8zuet7VZKuubKatpoKy6kbKaxo7H5ZFGyqsbeau0krKaRupiblTSLtWfQl442BHU3UfgedFAzwkH8aUowIcrBbOISJxSUjoXr00f3XvbSGMLZdUN0fBupKy6gfKazjDfc7SW1/b0vPtainHuC94+8o4dded1eRwk6PcNUm/FLQrmk7DWcu+99/Lcc89hjOH+++/nhhtu4NChQ9xwww1UV1fT0tLCgw8+yIc//GE+/elPs3HjRowxfOpTn+ILX/iC210QEReFg37CeWEm54V7bdfY0toZ2NWNlNc0dDwuiz7edrCao5FG2k6cRWdEeiBmFB46YUr9cG0bVfXNug48gXg2mL/xv9t452D1gL7nzLFZfP1js+Jq+/TTT7Nlyxa2bt3K0aNHWbBgAYsWLeKJJ57gsssu45//+Z9pbW3lyJEjbNmyhQMHDvD2228DUFlZOaB1i0jyCvp9jB+ZzviRJz8HDtDaZqmobQ/vxhOn02saeX3PMcprTtzE5St/+QMBnzPaz8kIkhNOJTccdJ6HU8mNHhuV4RzPCadqQZuL9Js/ifXr13PTTTfh8/koKCjggx/8IBs2bGDBggV86lOform5mauvvpopU6aQlpbG7t27+dznPscVV1zBkiVL+v4AEZF+8KWY6Ii498u4rLVU17d0hPVLr28hf8JkKmqbqIg0cqy2iaORJvZW1FIRaerxXDg4G7o4QZ3q7PyWkcqomBBvP9Ye6JpSHzieDeZ4R7aDxdoe5oyARYsWsW7dOn7/+99z8803c88993DnnXeydetWXnjhBR5++GFWr17No48+OsQVi4g4C9my0wNkpweYWpBJc6mfog+cuAtbu/qmVipqG6mINFFR28jRSBPHoiFeEWniaG0TZTUNbD9UTUWk6YTReLvMkL9zFJ7hBHduNLRzwkFyo99HZaQyMj2gu5P1wrPB7LZFixbx4x//mGXLlnHs2DHWrVvHAw88wL59+xg3bhy33347tbW1HVPdqampfPzjH2fKlCnceuutbpcvIhKXtFQf41P7nkoHZ8BS09hCRaSJY9EQr4hEQ7y2qWNUvq+ijjfer+RYbc/nxY2BkelOgMdOn+dkBKOj8uiIPDpCz0obXlE1vHrbD9dccw2vvPIKc+bMwRjDv//7vzN69Ggef/xxHnjgAQKBAOFwmB/+8IccOHCA5cuX09bm/J/kv/3bv7lcvYjIwDPGkBUKkBUKMCk3o8/27ZeXdQR3zKi8fVq9ItLE9sPOaLyq/sQV6gD+FEO6H/LfeIkRaQFGpAfITkslO/rYeR5wLnlLC3QczwwFEvKyMwVzN+3XMBtjeOCBB3jggQe6vL5s2TKWLVvW8bx9S8433nhjSOsUEfG62MvLpsbRvrm1jePRc+AVtZ3nw49GGnmnZB8ZI8NU1jVzsLKB7YdqqKxrovYk58jBGZlnhTqDOza8TziWHnBCPXrczXPmCmYREfGEgC/Fucyrh33Ki4sPU1Q074TjTS1tVDc0U1nXTFV9E5V17Y+bqaxvpqquicp651hlfTOlx+uprHNG5z1Ns7dLC/hiRuIBRsSM0LOjz7uGuxPwGam+074sTcEsIiIJK9WfQm44SG442K+fa2tzzpdXdYR4U0d4V9U1dQv3ZnYfjXS83tO+6u38KYYR6QGy0gLRkXlqx0h8RFpqXLUpmEVEZNhJSTEdo93+amhujYZ05wi9+oRwd4K9rKaBnUdqqKprpqaxJa73VzCLiIj0QyjgY3S2j9HZ/bs1aHNrG6nf7budLiQTEREZAoE4r91WMIuIiHiIgllERMRDFMwuaWmJbxGAiIgMLwrmHlx99dXMmzePWbNmsXLlSgCef/55zjvvPObMmcOll14KOJuRfOYzn+Gcc85h9uzZPPXUUwCEw523eXvyySc7tui89dZb+eIXv8jixYv58pe/zOuvv87ChQuZO3cuCxcuZMeOHQC0trbypS99qeN9f/CDH/CnP/2Ja665puN9//jHP3LttdcOxa9DRESGkHdXZT/3FTj81sC+5+hz4PLv9Nns0UcfZdSoUdTX17NgwQKuuuoqbr/9dtatW8ekSZM4duwYAN/85jfJysrirbecOo8fP97ne+/cuZO1a9fi8/morq5m3bp1+P1+1q5dy1e/+lWeeuopVq5cyZ49e9i8eTN+v59jx44xcuRI7r77bsrLy8nLy+OnP/0py5cvP73fh4iIeI53g9lF3//+93nmmWcA2L9/PytXrmTRokVMmjQJgFGjRgGwdu1aVq1a1fFzI0eO7PO9r7/+enw+Z6u3qqoqli1bxnvvvYcxhubm5o73veuuu/D7/V0+7+abb+YXv/gFy5cv55VXXuFnP/vZAPVYRES8wrvBHMfIdjAUFxezdu1aXnnlFdLT0ykqKmLOnDkd08yxrLU9br0We6yhoaHLaxkZnRu/f+1rX2Px4sU888wz7N27l6Kiol7fd/ny5XzsYx8jFApx/fXXdwS3iIgkD51j7qaqqoqRI0eSnp7Ou+++y6uvvkpjYyMvvfQSe/bsAeiYyl6yZEnHOWjonMouKChg+/bttLW1dYy8T/ZZ48aNA+Cxxx7rOL5kyRJWrFjRsUCs/fPGjh3L2LFj+da3vqVbS4qIJCkFczdLly6lpaWF2bNn87WvfY0LL7yQvLw8Vq5cybXXXsucOXO44YYbALj//vuprKzk7LPPZs6cObz44osAfOc73+HKK6/kQx/6EGPGjDnpZ917773cd999XHzxxbS2dt4h5bbbbmPixInMnj2bOXPm8MQTT3S89slPfpIJEyYwc+bMQfoNiIiImzQX2k0wGOS5557r8bXLL7+8y/NwOMyPf/xjMjMzuxy/7rrruO666074+dhRMcBFF13Ezp07O55/85vfBMDv9/PQQw/x0EMPnfAe69ev5/bbb4+rLyIikngUzAlk3rx5ZGRk8OCDD7pdioiIDBIFcwLZtGmT2yWIiMgg0zlmERERD/FcMFtr3S4hYel3JyKS+DwVzKFQiIqKCgXMKbDWUlFRQSjUv/uDioiIt3jqHPP48eMpLS2lvLzc7VLi1tDQ4JkwDIVCjB8/3u0yRETkNMQVzMaYpcD3AB+wylr7nW6vm+jrHwXqgFuttW/0t5hAINCx7WWiKC4uZu7cuW6XISIiSaLPqWxjjA94GLgcmAncZIzpvrvF5cDU6NcdwI8GuE4REZFhIZ5zzOcDJdba3dbaJuDXwFXd2lwF/Mw6XgVGGGNOvuWViIiI9CieYB4H7I95Xho91t82IiIi0od4zjGfeJsj6L5sOp42GGPuwJnqBmg0xrwdx+d7XS5w1O0iTlMy9AHUDy9Jhj5AcvQjGfoAydOPaX01iCeYS4EJMc/HAwdPoQ3W2pXASgBjzEZr7fw4Pt/TkqEfydAHUD+8JBn6AMnRj2ToAyRXP/pqE89U9gZgqjFmkjEmFbgReLZbm2eBW4zjQqDKWnuo3xWLiIgMc32OmK21LcaYe4AXcC6XetRau80Yc1f09RXAGpxLpUpwLpdaPngli4iIJK+4rmO21q7BCd/YYytiHlvg7n5+9sp+tveqZOhHMvQB1A8vSYY+QHL0Ixn6AMOoH0bbX4qIiHiHp/bKFhERGe5cCWZjzFJjzA5jTIkx5itu1HC6jDGPGmPKEvmSL2PMBGPMi8aY7caYbcaYv3e7plNhjAkZY143xmyN9uMbbtd0qowxPmPMZmPM79yu5VQZY/YaY94yxmyJZwWqFxljRhhjnjTGvBv9+3GR2zX1lzFmWvTPoP2r2hjzD27X1V/GmC9E/16/bYz5lTHGGzcn6CdjzN9H+7Ctrz+HIZ/Kjm7xuRP4CM5lVhuAm6y17wxpIafJGLMIiODseHa22/WciujubGOstW8YYzKBTcDVCfhnYYAMa23EGBMA1gN/H92FLqEYY74IzAeyrLVXul3PqTDG7AXmW2sT9ppTY8zjwF+stauiV6OkW2sr3a7rVEX/3T0AXGCt3ed2PfEyxozD+fs801pbb4xZDayx1j7mbmX9Y4w5G2fXzPOBJuB54DPW2vd6au/GiDmeLT49z1q7Djjmdh2nw1p7qP1mI9baGmA7CbhjW3Qr2Ej0aSD6lXCLJ4wx44ErgFVu1zKcGWOygEXAIwDW2qZEDuWoS4FdiRTKMfxAmjHGD6TTwx4ZCWAG8Kq1ts5a2wK8BFxzssZuBLO27/QgY8yZwFzgNXcrOTXRKeAtQBnwR2ttIvbjP4F7gTa3CzlNFviDMWZTdLe/RDMZKAd+Gj2tsMoYk+F2UafpRuBXbhfRX9baA8D/A94HDuHskfEHd6s6JW8Di4wxOcaYdJzLiyecrLEbwRzX9p0ydIwxYeAp4B+stdVu13MqrLWt1tpzcXadOz86dZQwjDFXAmXW2k1u1zIALrbWnodz17m7o6d9EokfOA/4kbV2LlALJORaGIDoVPzfAL9xu5b+MsaMxJlRnQSMBTKMMX/nblX9Z63dDnwX+CPONPZWoOVk7d0I5ri275ShET0n+xTwS2vt027Xc7qiU47FwFKXS+mvi4G/iZ6f/TXwIWPML9wt6dRYaw9Gv5cBz+CcvkokpUBpzKzLkzhBnaguB96w1h5xu5BT8GFgj7W23FrbDDwNLHS5plNirX3EWnuetXYRzmnQHs8vgzvBHM8WnzIEooumHgG2W2sfcrueU2WMyTPGjIg+TsP5y/yuu1X1j7X2PmvteGvtmTh/J/5srU24kYExJiO6kJDo9O8SnGm8hGGtPQzsN8a032zgUiChFkR2cxMJOI0d9T5woTEmPfrv1aU4a2ESjjEmP/p9InAtvfyZxLXz10A62RafQ13H6TLG/AooAnKNMaXA1621j7hbVb9dDNwMvBU9Pwvw1ehOb4lkDPB4dOVpCrDaWpuwlxsluALgGeffUPzAE9ba590t6ZR8DvhldPCwmwTdZjh6PvMjwJ1u13IqrLWvGWOeBN7AmfrdTOLuAPaUMSYHaAbuttYeP1lD7fwlIiLiIdr5S0RExEMUzCIiIh6iYBYREfEQBbOIiIiHKJhFREQ8RMEsIiLiIQpmERERD1Ewi4iIeMj/B+3Jzew1isd2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 画图展示学习曲线图\n",
    "def plot_learning_curves(history):\n",
    "    pd.DataFrame(history.history).plot(figsize=(8,5))\n",
    "    plt.grid(True)\n",
    "    plt.gca().set_ylim(0, 1)\n",
    "    plt.show()\n",
    "    \n",
    "plot_learning_curves(history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
